{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 0.导入常用库"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 加入下面2行，可以使py代码文件中的修改即时生效\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/zongyi/anaconda3/lib/python3.6/importlib/_bootstrap.py:219: RuntimeWarning: numpy.dtype size changed, may indicate binary incompatibility. Expected 96, got 88\n",
      "  return f(*args, **kwds)\n",
      "/home/zongyi/anaconda3/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using TensorFlow backend.\n",
      "/home/zongyi/anaconda3/lib/python3.6/importlib/_bootstrap.py:219: RuntimeWarning: numpy.dtype size changed, may indicate binary incompatibility. Expected 96, got 88\n",
      "  return f(*args, **kwds)\n",
      "/home/zongyi/anaconda3/lib/python3.6/importlib/_bootstrap.py:219: RuntimeWarning: numpy.dtype size changed, may indicate binary incompatibility. Expected 96, got 88\n",
      "  return f(*args, **kwds)\n"
     ]
    }
   ],
   "source": [
    "from __future__ import print_function\n",
    "import keras\n",
    "from keras.layers import Dense, Conv2D, BatchNormalization, Activation\n",
    "from keras.layers import AveragePooling2D, Input, Flatten\n",
    "from keras.optimizers import Adam\n",
    "from keras.callbacks import ModelCheckpoint, LearningRateScheduler\n",
    "from keras.callbacks import ReduceLROnPlateau\n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "from keras.regularizers import l2\n",
    "from keras import backend as K\n",
    "from keras.models import Model\n",
    "from keras.datasets import cifar10\n",
    "import numpy as np\n",
    "import os"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1.数据准备"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Model parameter\n",
    "# ----------------------------------------------------------------------------\n",
    "#           |      | 200-epoch | Orig Paper| 200-epoch | Orig Paper| sec/epoch\n",
    "# Model     |  n   | ResNet v1 | ResNet v1 | ResNet v2 | ResNet v2 | GTX1080Ti\n",
    "#           |v1(v2)| %Accuracy | %Accuracy | %Accuracy | %Accuracy | v1 (v2)\n",
    "# ----------------------------------------------------------------------------\n",
    "# ResNet20  | 3 (2)| 92.16     | 91.25     | -----     | -----     | 35 (---)\n",
    "# ResNet32  | 5(NA)| 92.46     | 92.49     | NA        | NA        | 50 ( NA)\n",
    "# ResNet44  | 7(NA)| 92.50     | 92.83     | NA        | NA        | 70 ( NA)\n",
    "# ResNet56  | 9 (6)| 92.71     | 93.03     | 93.01     | NA        | 90 (100)\n",
    "# ResNet110 |18(12)| 92.65     | 93.39+-.16| 93.15     | 93.63     | 165(180)\n",
    "# ResNet164 |27(18)| -----     | 94.07     | -----     | 94.54     | ---(---)\n",
    "# ResNet1001| (111)| -----     | 92.39     | -----     | 95.08+-.14| ---(---)\n",
    "# ---------------------------------------------------------------------------"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.1 定义函数load_train_dataset、load_test_dataset，用于加载数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.datasets.cifar import load_batch\n",
    "\n",
    "# 加载数据集cifar10里面的训练集\n",
    "def load_train_dataset(dirPath='../resources/cifar-10-batches-py/'):\n",
    "    train_sample_quantity = 50000\n",
    "    image_width = 32\n",
    "    image_height = 32\n",
    "    channel_quantity = 3\n",
    "    train_x = np.zeros((train_sample_quantity, channel_quantity, image_width, image_height),\n",
    "                       dtype='uint8')\n",
    "    train_y = np.zeros((train_sample_quantity, ),\n",
    "                       dtype='uint8')\n",
    "    for i in range(1, 6):\n",
    "        fileName = 'data_batch_%d' %i\n",
    "        filePath = os.path.join(dirPath, fileName)\n",
    "        startIndex = (i - 1) * 10000\n",
    "        endIndex = i * 10000\n",
    "        train_x[startIndex:endIndex, :, :, :], train_y[startIndex:endIndex] = load_batch(filePath)\n",
    "    print('未做矩阵转置：', train_x.shape)\n",
    "    # 从官网上下载的数据集的4个维度为样本个数n、通道数c、宽度w、高度h\n",
    "    # Keras基于Tensorflow，数据的维度顺序要求：样本个数n、宽度w、高度h、通道数c，所以使用np.transpose完成矩阵转置\n",
    "    train_x = train_x.transpose(0, 2, 3, 1)\n",
    "    print('矩阵转置后：', train_x.shape)\n",
    "    return train_x, train_y\n",
    "\n",
    "# 加载数据集cifar10里面的测试集\n",
    "def load_test_dataset(dirPath='../resources/cifar-10-batches-py/'):\n",
    "    fileName = 'test_batch'\n",
    "    filePath = os.path.join(dirPath, fileName)\n",
    "    test_x, test_y = load_batch(filePath)\n",
    "    print('未做矩阵转置：', test_x.shape)\n",
    "    test_x = test_x.transpose(0, 2, 3, 1)\n",
    "    print('矩阵转置后：', test_x.shape)\n",
    "    return test_x, test_y"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.2 对加载后的图像数据做处理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "未做矩阵转置： (50000, 3, 32, 32)\n",
      "矩阵转置后： (50000, 32, 32, 3)\n",
      "未做矩阵转置： (10000, 3, 32, 32)\n",
      "矩阵转置后： (10000, 32, 32, 3)\n",
      "x_train shape: (50000, 32, 32, 3)\n",
      "y_train shape: (50000,)\n",
      "50000 train samples\n",
      "10000 test samples\n",
      "未做to_categorical之前, y_train shape: (50000,)\n",
      "做to_categorical之后, y_train shape: (50000, 10)\n"
     ]
    }
   ],
   "source": [
    "# Subtracting pixel mean improves accuracy\n",
    "subtract_pixel_mean = True\n",
    "\n",
    "# Load the CIFAR10 data.\n",
    "dirPath = '../resources/cifar-10-batches-py/'\n",
    "x_train, y_train = load_train_dataset(dirPath)\n",
    "x_test, y_test = load_test_dataset(dirPath)\n",
    "\n",
    "# Input image dimensions.\n",
    "input_shape = x_train.shape[1:]\n",
    "\n",
    "# Normalize data.\n",
    "x_train = x_train.astype('float32') / 255\n",
    "x_test = x_test.astype('float32') / 255\n",
    "\n",
    "# If subtract pixel mean is enabled\n",
    "if subtract_pixel_mean:\n",
    "    x_train_mean = np.mean(x_train, axis=0)\n",
    "    x_train -= x_train_mean\n",
    "    x_test -= x_train_mean\n",
    "\n",
    "print('x_train shape:', x_train.shape)\n",
    "print('y_train shape:', y_train.shape)\n",
    "print(x_train.shape[0], 'train samples')\n",
    "print(x_test.shape[0], 'test samples')\n",
    "\n",
    "# Convert class vectors to binary class matrices.\n",
    "num_classes = 10\n",
    "print('未做to_categorical之前, y_train shape:', y_train.shape)\n",
    "y_train = keras.utils.to_categorical(y_train, num_classes)\n",
    "print('做to_categorical之后, y_train shape:', y_train.shape)\n",
    "y_test = keras.utils.to_categorical(y_test, num_classes)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.搭建神经网络"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.1 定义函数resnet_layer，返回值是经过resnet_layer计算的结果"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def resnet_layer(inputs,\n",
    "                 num_filters=16,\n",
    "                 kernel_size=3,\n",
    "                 strides=1,\n",
    "                 activation='relu',\n",
    "                 batch_normalization=True,\n",
    "                 conv_first=True):\n",
    "    \"\"\"2D Convolution-Batch Normalization-Activation stack builder\n",
    "    # Arguments\n",
    "        inputs (tensor): input tensor from input image or previous layer\n",
    "        num_filters (int): Conv2D number of filters\n",
    "        kernel_size (int): Conv2D square kernel dimensions\n",
    "        strides (int): Conv2D square stride dimensions\n",
    "        activation (string): activation name\n",
    "        batch_normalization (bool): whether to include batch normalization\n",
    "        conv_first (bool): conv-bn-activation (True) or\n",
    "            bn-activation-conv (False)\n",
    "    # Returns\n",
    "        x (tensor): tensor as input to the next layer\n",
    "    \"\"\"\n",
    "    conv = Conv2D(num_filters,\n",
    "                  kernel_size=kernel_size,\n",
    "                  strides=strides,\n",
    "                  padding='same',\n",
    "                  kernel_initializer='he_normal',\n",
    "                  kernel_regularizer=l2(1e-4))\n",
    "\n",
    "    x = inputs\n",
    "    if conv_first:\n",
    "        x = conv(x)\n",
    "        if batch_normalization:\n",
    "            x = BatchNormalization()(x)\n",
    "        if activation is not None:\n",
    "            x = Activation(activation)(x)\n",
    "    else:\n",
    "        if batch_normalization:\n",
    "            x = BatchNormalization()(x)\n",
    "        if activation is not None:\n",
    "            x = Activation(activation)(x)\n",
    "        x = conv(x)\n",
    "    return x"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.2 定义函数resnet_v1，返回值是模型对象"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def resnet_v1(input_shape, depth, num_classes=10):\n",
    "    \"\"\"ResNet Version 1 Model builder [a]\n",
    "    Stacks of 2 x (3 x 3) Conv2D-BN-ReLU\n",
    "    Last ReLU is after the shortcut connection.\n",
    "    At the beginning of each stage, the feature map size is halved (downsampled)\n",
    "    by a convolutional layer with strides=2, while the number of filters is\n",
    "    doubled. Within each stage, the layers have the same number filters and the\n",
    "    same number of filters.\n",
    "    Features maps sizes:\n",
    "    stage 0: 32x32, 16\n",
    "    stage 1: 16x16, 32\n",
    "    stage 2:  8x8,  64\n",
    "    The Number of parameters is approx the same as Table 6 of [a]:\n",
    "    ResNet20 0.27M\n",
    "    ResNet32 0.46M\n",
    "    ResNet44 0.66M\n",
    "    ResNet56 0.85M\n",
    "    ResNet110 1.7M\n",
    "    # Arguments\n",
    "        input_shape (tensor): shape of input image tensor\n",
    "        depth (int): number of core convolutional layers\n",
    "        num_classes (int): number of classes (CIFAR10 has 10)\n",
    "    # Returns\n",
    "        model (Model): Keras model instance\n",
    "    \"\"\"\n",
    "    if (depth - 2) % 6 != 0:\n",
    "        raise ValueError('depth should be 6n+2 (eg 20, 32, 44 in [a])')\n",
    "    # Start model definition.\n",
    "    num_filters = 16\n",
    "    num_res_blocks = int((depth - 2) / 6)\n",
    "\n",
    "    inputs = Input(shape=input_shape)\n",
    "    x = resnet_layer(inputs=inputs)\n",
    "    # Instantiate the stack of residual units\n",
    "    for stack in range(3):\n",
    "        for res_block in range(num_res_blocks):\n",
    "            strides = 1\n",
    "            if stack > 0 and res_block == 0:  # first layer but not first stack\n",
    "                strides = 2  # downsample\n",
    "            y = resnet_layer(inputs=x,\n",
    "                             num_filters=num_filters,\n",
    "                             strides=strides)\n",
    "            y = resnet_layer(inputs=y,\n",
    "                             num_filters=num_filters,\n",
    "                             activation=None)\n",
    "            if stack > 0 and res_block == 0:  # first layer but not first stack\n",
    "                # linear projection residual shortcut connection to match\n",
    "                # changed dims\n",
    "                x = resnet_layer(inputs=x,\n",
    "                                 num_filters=num_filters,\n",
    "                                 kernel_size=1,\n",
    "                                 strides=strides,\n",
    "                                 activation=None,\n",
    "                                 batch_normalization=False)\n",
    "            x = keras.layers.add([x, y])\n",
    "            x = Activation('relu')(x)\n",
    "        num_filters *= 2\n",
    "\n",
    "    # Add classifier on top.\n",
    "    # v1 does not use BN after last shortcut connection-ReLU\n",
    "    x = AveragePooling2D(pool_size=8)(x)\n",
    "    y = Flatten()(x)\n",
    "    outputs = Dense(num_classes,\n",
    "                    activation='softmax',\n",
    "                    kernel_initializer='he_normal')(y)\n",
    "\n",
    "    # Instantiate model.\n",
    "    model = Model(inputs=inputs, outputs=outputs)\n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.3 定义函数resnet_v2，返回值是模型对象"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def resnet_v2(input_shape, depth, num_classes=10):\n",
    "    \"\"\"ResNet Version 2 Model builder [b]\n",
    "    Stacks of (1 x 1)-(3 x 3)-(1 x 1) BN-ReLU-Conv2D or also known as\n",
    "    bottleneck layer\n",
    "    First shortcut connection per layer is 1 x 1 Conv2D.\n",
    "    Second and onwards shortcut connection is identity.\n",
    "    At the beginning of each stage, the feature map size is halved (downsampled)\n",
    "    by a convolutional layer with strides=2, while the number of filter maps is\n",
    "    doubled. Within each stage, the layers have the same number filters and the\n",
    "    same filter map sizes.\n",
    "    Features maps sizes:\n",
    "    conv1  : 32x32,  16\n",
    "    stage 0: 32x32,  64\n",
    "    stage 1: 16x16, 128\n",
    "    stage 2:  8x8,  256\n",
    "    # Arguments\n",
    "        input_shape (tensor): shape of input image tensor\n",
    "        depth (int): number of core convolutional layers\n",
    "        num_classes (int): number of classes (CIFAR10 has 10)\n",
    "    # Returns\n",
    "        model (Model): Keras model instance\n",
    "    \"\"\"\n",
    "    if (depth - 2) % 9 != 0:\n",
    "        raise ValueError('depth should be 9n+2 (eg 56 or 110 in [b])')\n",
    "    # Start model definition.\n",
    "    num_filters_in = 16\n",
    "    num_res_blocks = int((depth - 2) / 9)\n",
    "\n",
    "    inputs = Input(shape=input_shape)\n",
    "    # v2 performs Conv2D with BN-ReLU on input before splitting into 2 paths\n",
    "    x = resnet_layer(inputs=inputs,\n",
    "                     num_filters=num_filters_in,\n",
    "                     conv_first=True)\n",
    "\n",
    "    # Instantiate the stack of residual units\n",
    "    for stage in range(3):\n",
    "        for res_block in range(num_res_blocks):\n",
    "            activation = 'relu'\n",
    "            batch_normalization = True\n",
    "            strides = 1\n",
    "            if stage == 0:\n",
    "                num_filters_out = num_filters_in * 4\n",
    "                if res_block == 0:  # first layer and first stage\n",
    "                    activation = None\n",
    "                    batch_normalization = False\n",
    "            else:\n",
    "                num_filters_out = num_filters_in * 2\n",
    "                if res_block == 0:  # first layer but not first stage\n",
    "                    strides = 2    # downsample\n",
    "\n",
    "            # bottleneck residual unit\n",
    "            y = resnet_layer(inputs=x,\n",
    "                             num_filters=num_filters_in,\n",
    "                             kernel_size=1,\n",
    "                             strides=strides,\n",
    "                             activation=activation,\n",
    "                             batch_normalization=batch_normalization,\n",
    "                             conv_first=False)\n",
    "            y = resnet_layer(inputs=y,\n",
    "                             num_filters=num_filters_in,\n",
    "                             conv_first=False)\n",
    "            y = resnet_layer(inputs=y,\n",
    "                             num_filters=num_filters_out,\n",
    "                             kernel_size=1,\n",
    "                             conv_first=False)\n",
    "            if res_block == 0:\n",
    "                # linear projection residual shortcut connection to match\n",
    "                # changed dims\n",
    "                x = resnet_layer(inputs=x,\n",
    "                                 num_filters=num_filters_out,\n",
    "                                 kernel_size=1,\n",
    "                                 strides=strides,\n",
    "                                 activation=None,\n",
    "                                 batch_normalization=False)\n",
    "            x = keras.layers.add([x, y])\n",
    "\n",
    "        num_filters_in = num_filters_out\n",
    "\n",
    "    # Add classifier on top.\n",
    "    # v2 has BN-ReLU before Pooling\n",
    "    x = BatchNormalization()(x)\n",
    "    x = Activation('relu')(x)\n",
    "    x = AveragePooling2D(pool_size=8)(x)\n",
    "    y = Flatten()(x)\n",
    "    outputs = Dense(num_classes,\n",
    "                    activation='softmax',\n",
    "                    kernel_initializer='he_normal')(y)\n",
    "\n",
    "    # Instantiate model.\n",
    "    model = Model(inputs=inputs, outputs=outputs)\n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.4 实例化模型对象"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Model version\n",
    "# Orig paper: version = 1 (ResNet v1), Improved ResNet: version = 2 (ResNet v2)\n",
    "version = 2\n",
    "\n",
    "# Computed depth from supplied model parameter n\n",
    "n = 6\n",
    "if version == 1:\n",
    "    depth = n * 6 + 2\n",
    "elif version == 2:\n",
    "    depth = n * 9 + 2\n",
    "    \n",
    "# 根据ResNet版本，获取对应的模型对象\n",
    "if version == 2:\n",
    "    model = resnet_v2(input_shape=input_shape, depth=depth)\n",
    "else:\n",
    "    model = resnet_v1(input_shape=input_shape, depth=depth)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.5 多GPU并行训练"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from parallel_model import ParallelModel\n",
    "\n",
    "gpu_count = 2\n",
    "model = ParallelModel(model, gpu_count)\n",
    "model.compile(loss='categorical_crossentropy',\n",
    "              optimizer=Adam(lr=0.001),\n",
    "              metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.6 打印模型架构信息"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ResNet56v2\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            (None, 32, 32, 3)    0                                            \n",
      "__________________________________________________________________________________________________\n",
      "lambda_1 (Lambda)               (None, 32, 32, 3)    0           input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "lambda_2 (Lambda)               (None, 32, 32, 3)    0           input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "model_1 (Model)                 (None, 10)           1673738     lambda_1[0][0]                   \n",
      "                                                                 lambda_2[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Concatenate)           (None, 10)           0           model_1[1][0]                    \n",
      "                                                                 model_1[2][0]                    \n",
      "==================================================================================================\n",
      "Total params: 1,673,738\n",
      "Trainable params: 1,663,338\n",
      "Non-trainable params: 10,400\n",
      "__________________________________________________________________________________________________\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            (None, 32, 32, 3)    0                                            \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_1 (Conv2D)               (None, 32, 32, 16)   448         input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_1 (BatchNor (None, 32, 32, 16)   64          conv2d_1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "activation_1 (Activation)       (None, 32, 32, 16)   0           batch_normalization_1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_2 (Conv2D)               (None, 32, 32, 16)   272         activation_1[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_2 (BatchNor (None, 32, 32, 16)   64          conv2d_2[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "activation_2 (Activation)       (None, 32, 32, 16)   0           batch_normalization_2[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_3 (Conv2D)               (None, 32, 32, 16)   2320        activation_2[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_3 (BatchNor (None, 32, 32, 16)   64          conv2d_3[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "activation_3 (Activation)       (None, 32, 32, 16)   0           batch_normalization_3[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_5 (Conv2D)               (None, 32, 32, 64)   1088        activation_1[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_4 (Conv2D)               (None, 32, 32, 64)   1088        activation_3[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "add_1 (Add)                     (None, 32, 32, 64)   0           conv2d_5[0][0]                   \n",
      "                                                                 conv2d_4[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_4 (BatchNor (None, 32, 32, 64)   256         add_1[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_4 (Activation)       (None, 32, 32, 64)   0           batch_normalization_4[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_6 (Conv2D)               (None, 32, 32, 16)   1040        activation_4[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_5 (BatchNor (None, 32, 32, 16)   64          conv2d_6[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "activation_5 (Activation)       (None, 32, 32, 16)   0           batch_normalization_5[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_7 (Conv2D)               (None, 32, 32, 16)   2320        activation_5[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_6 (BatchNor (None, 32, 32, 16)   64          conv2d_7[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "activation_6 (Activation)       (None, 32, 32, 16)   0           batch_normalization_6[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_8 (Conv2D)               (None, 32, 32, 64)   1088        activation_6[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "add_2 (Add)                     (None, 32, 32, 64)   0           add_1[0][0]                      \n",
      "                                                                 conv2d_8[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_7 (BatchNor (None, 32, 32, 64)   256         add_2[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_7 (Activation)       (None, 32, 32, 64)   0           batch_normalization_7[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_9 (Conv2D)               (None, 32, 32, 16)   1040        activation_7[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_8 (BatchNor (None, 32, 32, 16)   64          conv2d_9[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "activation_8 (Activation)       (None, 32, 32, 16)   0           batch_normalization_8[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_10 (Conv2D)              (None, 32, 32, 16)   2320        activation_8[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_9 (BatchNor (None, 32, 32, 16)   64          conv2d_10[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_9 (Activation)       (None, 32, 32, 16)   0           batch_normalization_9[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_11 (Conv2D)              (None, 32, 32, 64)   1088        activation_9[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "add_3 (Add)                     (None, 32, 32, 64)   0           add_2[0][0]                      \n",
      "                                                                 conv2d_11[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_10 (BatchNo (None, 32, 32, 64)   256         add_3[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_10 (Activation)      (None, 32, 32, 64)   0           batch_normalization_10[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_12 (Conv2D)              (None, 32, 32, 16)   1040        activation_10[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_11 (BatchNo (None, 32, 32, 16)   64          conv2d_12[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_11 (Activation)      (None, 32, 32, 16)   0           batch_normalization_11[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_13 (Conv2D)              (None, 32, 32, 16)   2320        activation_11[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_12 (BatchNo (None, 32, 32, 16)   64          conv2d_13[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_12 (Activation)      (None, 32, 32, 16)   0           batch_normalization_12[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_14 (Conv2D)              (None, 32, 32, 64)   1088        activation_12[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "add_4 (Add)                     (None, 32, 32, 64)   0           add_3[0][0]                      \n",
      "                                                                 conv2d_14[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_13 (BatchNo (None, 32, 32, 64)   256         add_4[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_13 (Activation)      (None, 32, 32, 64)   0           batch_normalization_13[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_15 (Conv2D)              (None, 32, 32, 16)   1040        activation_13[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_14 (BatchNo (None, 32, 32, 16)   64          conv2d_15[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_14 (Activation)      (None, 32, 32, 16)   0           batch_normalization_14[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_16 (Conv2D)              (None, 32, 32, 16)   2320        activation_14[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_15 (BatchNo (None, 32, 32, 16)   64          conv2d_16[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_15 (Activation)      (None, 32, 32, 16)   0           batch_normalization_15[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_17 (Conv2D)              (None, 32, 32, 64)   1088        activation_15[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "add_5 (Add)                     (None, 32, 32, 64)   0           add_4[0][0]                      \n",
      "                                                                 conv2d_17[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_16 (BatchNo (None, 32, 32, 64)   256         add_5[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_16 (Activation)      (None, 32, 32, 64)   0           batch_normalization_16[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_18 (Conv2D)              (None, 32, 32, 16)   1040        activation_16[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_17 (BatchNo (None, 32, 32, 16)   64          conv2d_18[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_17 (Activation)      (None, 32, 32, 16)   0           batch_normalization_17[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_19 (Conv2D)              (None, 32, 32, 16)   2320        activation_17[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_18 (BatchNo (None, 32, 32, 16)   64          conv2d_19[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_18 (Activation)      (None, 32, 32, 16)   0           batch_normalization_18[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_20 (Conv2D)              (None, 32, 32, 64)   1088        activation_18[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "add_6 (Add)                     (None, 32, 32, 64)   0           add_5[0][0]                      \n",
      "                                                                 conv2d_20[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_19 (BatchNo (None, 32, 32, 64)   256         add_6[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_19 (Activation)      (None, 32, 32, 64)   0           batch_normalization_19[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_21 (Conv2D)              (None, 16, 16, 64)   4160        activation_19[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_20 (BatchNo (None, 16, 16, 64)   256         conv2d_21[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_20 (Activation)      (None, 16, 16, 64)   0           batch_normalization_20[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_22 (Conv2D)              (None, 16, 16, 64)   36928       activation_20[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_21 (BatchNo (None, 16, 16, 64)   256         conv2d_22[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_21 (Activation)      (None, 16, 16, 64)   0           batch_normalization_21[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_24 (Conv2D)              (None, 16, 16, 128)  8320        add_6[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_23 (Conv2D)              (None, 16, 16, 128)  8320        activation_21[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "add_7 (Add)                     (None, 16, 16, 128)  0           conv2d_24[0][0]                  \n",
      "                                                                 conv2d_23[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_22 (BatchNo (None, 16, 16, 128)  512         add_7[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_22 (Activation)      (None, 16, 16, 128)  0           batch_normalization_22[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_25 (Conv2D)              (None, 16, 16, 64)   8256        activation_22[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_23 (BatchNo (None, 16, 16, 64)   256         conv2d_25[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_23 (Activation)      (None, 16, 16, 64)   0           batch_normalization_23[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_26 (Conv2D)              (None, 16, 16, 64)   36928       activation_23[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_24 (BatchNo (None, 16, 16, 64)   256         conv2d_26[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_24 (Activation)      (None, 16, 16, 64)   0           batch_normalization_24[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_27 (Conv2D)              (None, 16, 16, 128)  8320        activation_24[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "add_8 (Add)                     (None, 16, 16, 128)  0           add_7[0][0]                      \n",
      "                                                                 conv2d_27[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_25 (BatchNo (None, 16, 16, 128)  512         add_8[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_25 (Activation)      (None, 16, 16, 128)  0           batch_normalization_25[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_28 (Conv2D)              (None, 16, 16, 64)   8256        activation_25[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_26 (BatchNo (None, 16, 16, 64)   256         conv2d_28[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_26 (Activation)      (None, 16, 16, 64)   0           batch_normalization_26[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_29 (Conv2D)              (None, 16, 16, 64)   36928       activation_26[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_27 (BatchNo (None, 16, 16, 64)   256         conv2d_29[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_27 (Activation)      (None, 16, 16, 64)   0           batch_normalization_27[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_30 (Conv2D)              (None, 16, 16, 128)  8320        activation_27[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "add_9 (Add)                     (None, 16, 16, 128)  0           add_8[0][0]                      \n",
      "                                                                 conv2d_30[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_28 (BatchNo (None, 16, 16, 128)  512         add_9[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_28 (Activation)      (None, 16, 16, 128)  0           batch_normalization_28[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_31 (Conv2D)              (None, 16, 16, 64)   8256        activation_28[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_29 (BatchNo (None, 16, 16, 64)   256         conv2d_31[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_29 (Activation)      (None, 16, 16, 64)   0           batch_normalization_29[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_32 (Conv2D)              (None, 16, 16, 64)   36928       activation_29[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_30 (BatchNo (None, 16, 16, 64)   256         conv2d_32[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_30 (Activation)      (None, 16, 16, 64)   0           batch_normalization_30[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_33 (Conv2D)              (None, 16, 16, 128)  8320        activation_30[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "add_10 (Add)                    (None, 16, 16, 128)  0           add_9[0][0]                      \n",
      "                                                                 conv2d_33[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_31 (BatchNo (None, 16, 16, 128)  512         add_10[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "activation_31 (Activation)      (None, 16, 16, 128)  0           batch_normalization_31[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_34 (Conv2D)              (None, 16, 16, 64)   8256        activation_31[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_32 (BatchNo (None, 16, 16, 64)   256         conv2d_34[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_32 (Activation)      (None, 16, 16, 64)   0           batch_normalization_32[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_35 (Conv2D)              (None, 16, 16, 64)   36928       activation_32[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_33 (BatchNo (None, 16, 16, 64)   256         conv2d_35[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_33 (Activation)      (None, 16, 16, 64)   0           batch_normalization_33[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_36 (Conv2D)              (None, 16, 16, 128)  8320        activation_33[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "add_11 (Add)                    (None, 16, 16, 128)  0           add_10[0][0]                     \n",
      "                                                                 conv2d_36[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_34 (BatchNo (None, 16, 16, 128)  512         add_11[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "activation_34 (Activation)      (None, 16, 16, 128)  0           batch_normalization_34[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_37 (Conv2D)              (None, 16, 16, 64)   8256        activation_34[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_35 (BatchNo (None, 16, 16, 64)   256         conv2d_37[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_35 (Activation)      (None, 16, 16, 64)   0           batch_normalization_35[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_38 (Conv2D)              (None, 16, 16, 64)   36928       activation_35[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_36 (BatchNo (None, 16, 16, 64)   256         conv2d_38[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_36 (Activation)      (None, 16, 16, 64)   0           batch_normalization_36[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_39 (Conv2D)              (None, 16, 16, 128)  8320        activation_36[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "add_12 (Add)                    (None, 16, 16, 128)  0           add_11[0][0]                     \n",
      "                                                                 conv2d_39[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_37 (BatchNo (None, 16, 16, 128)  512         add_12[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "activation_37 (Activation)      (None, 16, 16, 128)  0           batch_normalization_37[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_40 (Conv2D)              (None, 8, 8, 128)    16512       activation_37[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_38 (BatchNo (None, 8, 8, 128)    512         conv2d_40[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_38 (Activation)      (None, 8, 8, 128)    0           batch_normalization_38[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_41 (Conv2D)              (None, 8, 8, 128)    147584      activation_38[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_39 (BatchNo (None, 8, 8, 128)    512         conv2d_41[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_39 (Activation)      (None, 8, 8, 128)    0           batch_normalization_39[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_43 (Conv2D)              (None, 8, 8, 256)    33024       add_12[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_42 (Conv2D)              (None, 8, 8, 256)    33024       activation_39[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "add_13 (Add)                    (None, 8, 8, 256)    0           conv2d_43[0][0]                  \n",
      "                                                                 conv2d_42[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_40 (BatchNo (None, 8, 8, 256)    1024        add_13[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "activation_40 (Activation)      (None, 8, 8, 256)    0           batch_normalization_40[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_44 (Conv2D)              (None, 8, 8, 128)    32896       activation_40[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_41 (BatchNo (None, 8, 8, 128)    512         conv2d_44[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_41 (Activation)      (None, 8, 8, 128)    0           batch_normalization_41[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_45 (Conv2D)              (None, 8, 8, 128)    147584      activation_41[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_42 (BatchNo (None, 8, 8, 128)    512         conv2d_45[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_42 (Activation)      (None, 8, 8, 128)    0           batch_normalization_42[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_46 (Conv2D)              (None, 8, 8, 256)    33024       activation_42[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "add_14 (Add)                    (None, 8, 8, 256)    0           add_13[0][0]                     \n",
      "                                                                 conv2d_46[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_43 (BatchNo (None, 8, 8, 256)    1024        add_14[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "activation_43 (Activation)      (None, 8, 8, 256)    0           batch_normalization_43[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_47 (Conv2D)              (None, 8, 8, 128)    32896       activation_43[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_44 (BatchNo (None, 8, 8, 128)    512         conv2d_47[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_44 (Activation)      (None, 8, 8, 128)    0           batch_normalization_44[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_48 (Conv2D)              (None, 8, 8, 128)    147584      activation_44[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_45 (BatchNo (None, 8, 8, 128)    512         conv2d_48[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_45 (Activation)      (None, 8, 8, 128)    0           batch_normalization_45[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_49 (Conv2D)              (None, 8, 8, 256)    33024       activation_45[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "add_15 (Add)                    (None, 8, 8, 256)    0           add_14[0][0]                     \n",
      "                                                                 conv2d_49[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_46 (BatchNo (None, 8, 8, 256)    1024        add_15[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "activation_46 (Activation)      (None, 8, 8, 256)    0           batch_normalization_46[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_50 (Conv2D)              (None, 8, 8, 128)    32896       activation_46[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_47 (BatchNo (None, 8, 8, 128)    512         conv2d_50[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_47 (Activation)      (None, 8, 8, 128)    0           batch_normalization_47[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_51 (Conv2D)              (None, 8, 8, 128)    147584      activation_47[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_48 (BatchNo (None, 8, 8, 128)    512         conv2d_51[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_48 (Activation)      (None, 8, 8, 128)    0           batch_normalization_48[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_52 (Conv2D)              (None, 8, 8, 256)    33024       activation_48[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "add_16 (Add)                    (None, 8, 8, 256)    0           add_15[0][0]                     \n",
      "                                                                 conv2d_52[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_49 (BatchNo (None, 8, 8, 256)    1024        add_16[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "activation_49 (Activation)      (None, 8, 8, 256)    0           batch_normalization_49[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_53 (Conv2D)              (None, 8, 8, 128)    32896       activation_49[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_50 (BatchNo (None, 8, 8, 128)    512         conv2d_53[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_50 (Activation)      (None, 8, 8, 128)    0           batch_normalization_50[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_54 (Conv2D)              (None, 8, 8, 128)    147584      activation_50[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_51 (BatchNo (None, 8, 8, 128)    512         conv2d_54[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_51 (Activation)      (None, 8, 8, 128)    0           batch_normalization_51[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_55 (Conv2D)              (None, 8, 8, 256)    33024       activation_51[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "add_17 (Add)                    (None, 8, 8, 256)    0           add_16[0][0]                     \n",
      "                                                                 conv2d_55[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_52 (BatchNo (None, 8, 8, 256)    1024        add_17[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "activation_52 (Activation)      (None, 8, 8, 256)    0           batch_normalization_52[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_56 (Conv2D)              (None, 8, 8, 128)    32896       activation_52[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_53 (BatchNo (None, 8, 8, 128)    512         conv2d_56[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_53 (Activation)      (None, 8, 8, 128)    0           batch_normalization_53[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_57 (Conv2D)              (None, 8, 8, 128)    147584      activation_53[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_54 (BatchNo (None, 8, 8, 128)    512         conv2d_57[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_54 (Activation)      (None, 8, 8, 128)    0           batch_normalization_54[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_58 (Conv2D)              (None, 8, 8, 256)    33024       activation_54[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "add_18 (Add)                    (None, 8, 8, 256)    0           add_17[0][0]                     \n",
      "                                                                 conv2d_58[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_55 (BatchNo (None, 8, 8, 256)    1024        add_18[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "activation_55 (Activation)      (None, 8, 8, 256)    0           batch_normalization_55[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "average_pooling2d_1 (AveragePoo (None, 1, 1, 256)    0           activation_55[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "flatten_1 (Flatten)             (None, 256)          0           average_pooling2d_1[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 10)           2570        flatten_1[0][0]                  \n",
      "==================================================================================================\n",
      "Total params: 1,673,738\n",
      "Trainable params: 1,663,338\n",
      "Non-trainable params: 10,400\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Model name, depth and version\n",
    "\n",
    "model_type = 'ResNet%dv%d' % (depth, version)\n",
    "print(model_type)\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3.模型训练"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.1 规划学习率，训练到后期，学习率需要减少"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def lr_schedule(epoch):\n",
    "    \"\"\"Learning Rate Schedule\n",
    "    Learning rate is scheduled to be reduced after 80, 120, 160, 180 epochs.\n",
    "    Called automatically every epoch as part of callbacks during training.\n",
    "    # Arguments\n",
    "        epoch (int): The number of epochs\n",
    "    # Returns\n",
    "        lr (float32): learning rate\n",
    "    \"\"\"\n",
    "    lr = 1e-3\n",
    "    if epoch > 180:\n",
    "        lr *= 0.5e-3\n",
    "    elif epoch > 160:\n",
    "        lr *= 1e-3\n",
    "    elif epoch > 120:\n",
    "        lr *= 1e-2\n",
    "    elif epoch > 80:\n",
    "        lr *= 1e-1\n",
    "    print('Learning rate: ', lr)\n",
    "    return lr"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.2 模型训练时的参数设置"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Training parameters\n",
    "batch_size = 64  # orig paper trained all networks with batch_size=128\n",
    "epochs = 200\n",
    "\n",
    "# Prepare model model saving directory.\n",
    "save_dir = os.path.join(os.getcwd(), 'saved_models')\n",
    "model_name = 'cifar10_%s_model.{epoch:03d}.h5' % model_type\n",
    "if not os.path.isdir(save_dir):\n",
    "    os.makedirs(save_dir)\n",
    "filepath = os.path.join(save_dir, model_name)\n",
    "\n",
    "# Prepare callbacks for model saving and for learning rate adjustment.\n",
    "checkpoint = ModelCheckpoint(filepath=filepath,\n",
    "                             monitor='val_acc',\n",
    "                             verbose=0,\n",
    "                             save_best_only=True)\n",
    "lr_scheduler = LearningRateScheduler(lr_schedule)\n",
    "lr_reducer = ReduceLROnPlateau(factor=np.sqrt(0.1),\n",
    "                               cooldown=0,\n",
    "                               patience=5,\n",
    "                               min_lr=0.5e-6)\n",
    "callbacks = [checkpoint, lr_reducer, lr_scheduler]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.3 使用图像增强的结果做模型训练"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "data_augmentation = True\n",
    "if data_augmentation:\n",
    "    print('Using real-time data augmentation.')\n",
    "    # This will do preprocessing and realtime data augmentation:\n",
    "    datagen = ImageDataGenerator(\n",
    "        # set input mean to 0 over the dataset\n",
    "        featurewise_center=False,\n",
    "        # set each sample mean to 0\n",
    "        samplewise_center=False,\n",
    "        # divide inputs by std of dataset\n",
    "        featurewise_std_normalization=False,\n",
    "        # divide each input by its std\n",
    "        samplewise_std_normalization=False,\n",
    "        # apply ZCA whitening\n",
    "        zca_whitening=False,\n",
    "        # epsilon for ZCA whitening\n",
    "        zca_epsilon=1e-06,\n",
    "        # randomly rotate images in the range (deg 0 to 180)\n",
    "        rotation_range=0,\n",
    "        # randomly shift images horizontally\n",
    "        width_shift_range=0.1,\n",
    "        # randomly shift images vertically\n",
    "        height_shift_range=0.1,\n",
    "        # set range for random shear\n",
    "        shear_range=0.,\n",
    "        # set range for random zoom\n",
    "        zoom_range=0.,\n",
    "        # set range for random channel shifts\n",
    "        channel_shift_range=0.,\n",
    "        # set mode for filling points outside the input boundaries\n",
    "        fill_mode='nearest',\n",
    "        # value used for fill_mode = \"constant\"\n",
    "        cval=0.,\n",
    "        # randomly flip images\n",
    "        horizontal_flip=True,\n",
    "        # randomly flip images\n",
    "        vertical_flip=False,\n",
    "        # set rescaling factor (applied before any other transformation)\n",
    "        rescale=None,\n",
    "        # set function that will be applied on each input\n",
    "        preprocessing_function=None,\n",
    "        # image data format, either \"channels_first\" or \"channels_last\"\n",
    "        data_format=None,\n",
    "        # fraction of images reserved for validation (strictly between 0 and 1)\n",
    "        validation_split=0.0)\n",
    "\n",
    "    # Compute quantities required for featurewise normalization\n",
    "    # (std, mean, and principal components if ZCA whitening is applied).\n",
    "    datagen.fit(x_train)\n",
    "\n",
    "    # Fit the model on the batches generated by datagen.flow().\n",
    "    model.fit_generator(datagen.flow(x_train, y_train, batch_size=batch_size),\n",
    "                        validation_data=(x_test, y_test),\n",
    "                        epochs=epochs,\n",
    "                        verbose=1, \n",
    "                        workers=4,\n",
    "                        callbacks=callbacks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/200\n",
      "Learning rate:  0.001\n",
      "196/196 [==============================] - 24s 122ms/step - loss: 0.6848 - acc: 0.8637 - val_loss: 0.8849 - val_acc: 0.7977\n",
      "Epoch 2/200\n",
      "Learning rate:  0.001\n",
      "196/196 [==============================] - 24s 121ms/step - loss: 0.6588 - acc: 0.8678 - val_loss: 1.0683 - val_acc: 0.7365\n",
      "Epoch 3/200\n",
      "Learning rate:  0.001\n",
      "196/196 [==============================] - 24s 123ms/step - loss: 0.6402 - acc: 0.8734 - val_loss: 1.0007 - val_acc: 0.7722\n",
      "Epoch 4/200\n",
      "Learning rate:  0.001\n",
      "196/196 [==============================] - 24s 123ms/step - loss: 0.6248 - acc: 0.8756 - val_loss: 0.9805 - val_acc: 0.7801\n",
      "Epoch 5/200\n",
      "Learning rate:  0.001\n",
      "196/196 [==============================] - 24s 124ms/step - loss: 0.6166 - acc: 0.8772 - val_loss: 0.7843 - val_acc: 0.8224\n",
      "Epoch 6/200\n",
      "Learning rate:  0.001\n",
      "196/196 [==============================] - 24s 124ms/step - loss: 0.6067 - acc: 0.8795 - val_loss: 0.8058 - val_acc: 0.8148\n",
      "Epoch 7/200\n",
      "Learning rate:  0.001\n",
      "196/196 [==============================] - 24s 124ms/step - loss: 0.5927 - acc: 0.8847 - val_loss: 0.7014 - val_acc: 0.8462\n",
      "Epoch 8/200\n",
      "Learning rate:  0.001\n",
      "196/196 [==============================] - 24s 124ms/step - loss: 0.5844 - acc: 0.8866 - val_loss: 0.7293 - val_acc: 0.8434\n",
      "Epoch 9/200\n",
      "Learning rate:  0.001\n",
      "196/196 [==============================] - 25s 127ms/step - loss: 0.5744 - acc: 0.8892 - val_loss: 0.8880 - val_acc: 0.7981\n",
      "Epoch 10/200\n",
      "Learning rate:  0.001\n",
      "196/196 [==============================] - 25s 128ms/step - loss: 0.5668 - acc: 0.8891 - val_loss: 0.7732 - val_acc: 0.8276\n",
      "Epoch 11/200\n",
      "Learning rate:  0.001\n",
      "196/196 [==============================] - 25s 127ms/step - loss: 0.5554 - acc: 0.8922 - val_loss: 0.8913 - val_acc: 0.8035\n",
      "Epoch 12/200\n",
      "Learning rate:  0.001\n",
      "196/196 [==============================] - 25s 127ms/step - loss: 0.5468 - acc: 0.8958 - val_loss: 0.9440 - val_acc: 0.7854\n",
      "Epoch 13/200\n",
      "Learning rate:  0.001\n",
      "196/196 [==============================] - 25s 127ms/step - loss: 0.5375 - acc: 0.8973 - val_loss: 0.7658 - val_acc: 0.8228\n",
      "Epoch 14/200\n",
      "Learning rate:  0.001\n",
      "196/196 [==============================] - 25s 128ms/step - loss: 0.5286 - acc: 0.9010 - val_loss: 0.8087 - val_acc: 0.8208\n",
      "Epoch 15/200\n",
      "Learning rate:  0.001\n",
      "196/196 [==============================] - 25s 128ms/step - loss: 0.5285 - acc: 0.9008 - val_loss: 0.7334 - val_acc: 0.8363\n",
      "Epoch 16/200\n",
      "Learning rate:  0.001\n",
      "196/196 [==============================] - 25s 127ms/step - loss: 0.5147 - acc: 0.9047 - val_loss: 0.8278 - val_acc: 0.8155\n",
      "Epoch 17/200\n",
      "Learning rate:  0.001\n",
      "196/196 [==============================] - 25s 128ms/step - loss: 0.5138 - acc: 0.9050 - val_loss: 1.0779 - val_acc: 0.7655\n",
      "Epoch 18/200\n",
      "Learning rate:  0.001\n",
      "196/196 [==============================] - 25s 128ms/step - loss: 0.5060 - acc: 0.9066 - val_loss: 0.8759 - val_acc: 0.8159\n",
      "Epoch 19/200\n",
      "Learning rate:  0.001\n",
      "196/196 [==============================] - 25s 129ms/step - loss: 0.4977 - acc: 0.9087 - val_loss: 0.7298 - val_acc: 0.8452\n",
      "Epoch 20/200\n",
      "Learning rate:  0.001\n",
      "196/196 [==============================] - 25s 130ms/step - loss: 0.4877 - acc: 0.9135 - val_loss: 0.8018 - val_acc: 0.8256\n",
      "Epoch 21/200\n",
      "Learning rate:  0.001\n",
      "196/196 [==============================] - 25s 129ms/step - loss: 0.4844 - acc: 0.9137 - val_loss: 0.7981 - val_acc: 0.8246\n",
      "Epoch 22/200\n",
      "Learning rate:  0.001\n",
      "196/196 [==============================] - 25s 129ms/step - loss: 0.4789 - acc: 0.9153 - val_loss: 0.8538 - val_acc: 0.8164\n",
      "Epoch 23/200\n",
      "Learning rate:  0.001\n",
      "196/196 [==============================] - 25s 129ms/step - loss: 0.4767 - acc: 0.9149 - val_loss: 0.9254 - val_acc: 0.7999\n",
      "Epoch 24/200\n",
      "Learning rate:  0.001\n",
      "196/196 [==============================] - 26s 133ms/step - loss: 0.4648 - acc: 0.9200 - val_loss: 0.7719 - val_acc: 0.8402\n",
      "Epoch 25/200\n",
      "Learning rate:  0.001\n",
      "196/196 [==============================] - 25s 128ms/step - loss: 0.4619 - acc: 0.9199 - val_loss: 0.9189 - val_acc: 0.7985\n",
      "Epoch 26/200\n",
      "Learning rate:  0.001\n",
      "196/196 [==============================] - 25s 130ms/step - loss: 0.4626 - acc: 0.9184 - val_loss: 0.7074 - val_acc: 0.8469\n",
      "Epoch 27/200\n",
      "Learning rate:  0.001\n",
      "196/196 [==============================] - 24s 123ms/step - loss: 0.4594 - acc: 0.9202 - val_loss: 0.8621 - val_acc: 0.8123\n",
      "Epoch 28/200\n",
      "Learning rate:  0.001\n",
      "196/196 [==============================] - 24s 124ms/step - loss: 0.4497 - acc: 0.9239 - val_loss: 0.8421 - val_acc: 0.8188\n",
      "Epoch 29/200\n",
      "Learning rate:  0.001\n",
      "196/196 [==============================] - 24s 124ms/step - loss: 0.4430 - acc: 0.9256 - val_loss: 0.7873 - val_acc: 0.8292\n",
      "Epoch 30/200\n",
      "Learning rate:  0.001\n",
      "196/196 [==============================] - 24s 123ms/step - loss: 0.4408 - acc: 0.9259 - val_loss: 0.8602 - val_acc: 0.8161\n",
      "Epoch 31/200\n",
      "Learning rate:  0.001\n",
      "196/196 [==============================] - 24s 124ms/step - loss: 0.4410 - acc: 0.9253 - val_loss: 0.8550 - val_acc: 0.8122\n",
      "Epoch 32/200\n",
      "Learning rate:  0.001\n",
      "196/196 [==============================] - 24s 124ms/step - loss: 0.4293 - acc: 0.9290 - val_loss: 0.8424 - val_acc: 0.8195\n",
      "Epoch 33/200\n",
      "Learning rate:  0.001\n",
      "196/196 [==============================] - 24s 124ms/step - loss: 0.4289 - acc: 0.9294 - val_loss: 0.9935 - val_acc: 0.7900\n",
      "Epoch 34/200\n",
      "Learning rate:  0.001\n",
      "196/196 [==============================] - 24s 123ms/step - loss: 0.4310 - acc: 0.9284 - val_loss: 0.7041 - val_acc: 0.8477\n",
      "Epoch 35/200\n",
      "Learning rate:  0.001\n",
      "196/196 [==============================] - 24s 124ms/step - loss: 0.4222 - acc: 0.9299 - val_loss: 1.0628 - val_acc: 0.7862\n",
      "Epoch 36/200\n",
      "Learning rate:  0.001\n",
      "196/196 [==============================] - 24s 124ms/step - loss: 0.4211 - acc: 0.9307 - val_loss: 0.7352 - val_acc: 0.8492\n",
      "Epoch 37/200\n",
      "Learning rate:  0.001\n",
      "196/196 [==============================] - 24s 123ms/step - loss: 0.4208 - acc: 0.9312 - val_loss: 0.7234 - val_acc: 0.8538\n",
      "Epoch 38/200\n",
      "Learning rate:  0.001\n",
      "196/196 [==============================] - 24s 124ms/step - loss: 0.4133 - acc: 0.9327 - val_loss: 0.8299 - val_acc: 0.8338\n",
      "Epoch 39/200\n",
      "Learning rate:  0.001\n",
      "196/196 [==============================] - 24s 123ms/step - loss: 0.4094 - acc: 0.9337 - val_loss: 0.7044 - val_acc: 0.8554\n",
      "Epoch 40/200\n",
      "Learning rate:  0.001\n",
      "196/196 [==============================] - 24s 123ms/step - loss: 0.4107 - acc: 0.9343 - val_loss: 1.3864 - val_acc: 0.7209\n",
      "Epoch 41/200\n",
      "Learning rate:  0.001\n",
      "196/196 [==============================] - 24s 124ms/step - loss: 0.4014 - acc: 0.9360 - val_loss: 0.6854 - val_acc: 0.8562\n",
      "Epoch 42/200\n",
      "Learning rate:  0.001\n",
      "196/196 [==============================] - 24s 123ms/step - loss: 0.4103 - acc: 0.9334 - val_loss: 0.8198 - val_acc: 0.8313\n",
      "Epoch 43/200\n",
      "Learning rate:  0.001\n",
      "196/196 [==============================] - 24s 123ms/step - loss: 0.4020 - acc: 0.9366 - val_loss: 0.7052 - val_acc: 0.8519\n",
      "Epoch 44/200\n",
      "Learning rate:  0.001\n",
      "196/196 [==============================] - 24s 123ms/step - loss: 0.4006 - acc: 0.9371 - val_loss: 1.0050 - val_acc: 0.7954\n",
      "Epoch 45/200\n",
      "Learning rate:  0.001\n",
      "196/196 [==============================] - 24s 124ms/step - loss: 0.3879 - acc: 0.9399 - val_loss: 0.7030 - val_acc: 0.8600\n",
      "Epoch 46/200\n",
      "Learning rate:  0.001\n",
      "196/196 [==============================] - 24s 124ms/step - loss: 0.4047 - acc: 0.9339 - val_loss: 0.8156 - val_acc: 0.8311\n",
      "Epoch 47/200\n",
      "Learning rate:  0.001\n",
      "196/196 [==============================] - 24s 124ms/step - loss: 0.3867 - acc: 0.9404 - val_loss: 0.7516 - val_acc: 0.8435\n",
      "Epoch 48/200\n",
      "Learning rate:  0.001\n",
      "196/196 [==============================] - 24s 124ms/step - loss: 0.3828 - acc: 0.9413 - val_loss: 0.8385 - val_acc: 0.8297\n",
      "Epoch 49/200\n",
      "Learning rate:  0.001\n",
      "196/196 [==============================] - 24s 124ms/step - loss: 0.3796 - acc: 0.9428 - val_loss: 0.7235 - val_acc: 0.8463\n",
      "Epoch 50/200\n",
      "Learning rate:  0.001\n",
      "196/196 [==============================] - 24s 124ms/step - loss: 0.3875 - acc: 0.9402 - val_loss: 0.9999 - val_acc: 0.7979\n",
      "Epoch 51/200\n",
      "Learning rate:  0.001\n",
      "196/196 [==============================] - 24s 124ms/step - loss: 0.3865 - acc: 0.9396 - val_loss: 0.6934 - val_acc: 0.8586\n",
      "Epoch 52/200\n",
      "Learning rate:  0.001\n",
      "196/196 [==============================] - 24s 121ms/step - loss: 0.3766 - acc: 0.9434 - val_loss: 0.8503 - val_acc: 0.8283\n",
      "Epoch 53/200\n",
      "Learning rate:  0.001\n",
      "196/196 [==============================] - 24s 120ms/step - loss: 0.3767 - acc: 0.9425 - val_loss: 0.6786 - val_acc: 0.8675\n",
      "Epoch 54/200\n",
      "Learning rate:  0.001\n",
      "196/196 [==============================] - 23s 119ms/step - loss: 0.3759 - acc: 0.9425 - val_loss: 0.7357 - val_acc: 0.8516\n",
      "Epoch 55/200\n",
      "Learning rate:  0.001\n",
      "196/196 [==============================] - 23s 119ms/step - loss: 0.3712 - acc: 0.9453 - val_loss: 0.6621 - val_acc: 0.8679\n",
      "Epoch 56/200\n",
      "Learning rate:  0.001\n",
      "196/196 [==============================] - 23s 119ms/step - loss: 0.3667 - acc: 0.9460 - val_loss: 0.7582 - val_acc: 0.8521\n",
      "Epoch 57/200\n",
      "Learning rate:  0.001\n",
      "196/196 [==============================] - 23s 119ms/step - loss: 0.3688 - acc: 0.9446 - val_loss: 0.7614 - val_acc: 0.8385\n",
      "Epoch 58/200\n",
      "Learning rate:  0.001\n",
      "196/196 [==============================] - 23s 119ms/step - loss: 0.3688 - acc: 0.9440 - val_loss: 0.7993 - val_acc: 0.8348\n",
      "Epoch 59/200\n",
      "Learning rate:  0.001\n",
      "196/196 [==============================] - 23s 118ms/step - loss: 0.3723 - acc: 0.9445 - val_loss: 0.7852 - val_acc: 0.8440\n",
      "Epoch 60/200\n",
      "Learning rate:  0.001\n",
      "196/196 [==============================] - 23s 119ms/step - loss: 0.3632 - acc: 0.9468 - val_loss: 0.8176 - val_acc: 0.8420\n",
      "Epoch 61/200\n",
      "Learning rate:  0.001\n",
      "196/196 [==============================] - 23s 119ms/step - loss: 0.3609 - acc: 0.9476 - val_loss: 0.7560 - val_acc: 0.8587\n",
      "Epoch 62/200\n",
      "Learning rate:  0.001\n",
      "196/196 [==============================] - 23s 119ms/step - loss: 0.3600 - acc: 0.9476 - val_loss: 0.6592 - val_acc: 0.8779\n",
      "Epoch 63/200\n",
      "Learning rate:  0.001\n",
      "196/196 [==============================] - 23s 119ms/step - loss: 0.3599 - acc: 0.9466 - val_loss: 0.8717 - val_acc: 0.8341\n",
      "Epoch 64/200\n",
      "Learning rate:  0.001\n",
      "196/196 [==============================] - 23s 119ms/step - loss: 0.3554 - acc: 0.9488 - val_loss: 0.6982 - val_acc: 0.8630\n",
      "Epoch 65/200\n",
      "Learning rate:  0.001\n",
      "196/196 [==============================] - 23s 119ms/step - loss: 0.3635 - acc: 0.9460 - val_loss: 0.7284 - val_acc: 0.8448\n",
      "Epoch 66/200\n",
      "Learning rate:  0.001\n",
      "196/196 [==============================] - 23s 120ms/step - loss: 0.3581 - acc: 0.9482 - val_loss: 0.7637 - val_acc: 0.8443\n",
      "Epoch 67/200\n",
      "Learning rate:  0.001\n",
      "196/196 [==============================] - 23s 119ms/step - loss: 0.3513 - acc: 0.9492 - val_loss: 0.7466 - val_acc: 0.8480\n",
      "Epoch 68/200\n",
      "Learning rate:  0.001\n",
      "196/196 [==============================] - 23s 119ms/step - loss: 0.3588 - acc: 0.9470 - val_loss: 0.7337 - val_acc: 0.8577\n",
      "Epoch 69/200\n",
      "Learning rate:  0.001\n",
      "196/196 [==============================] - 23s 118ms/step - loss: 0.3560 - acc: 0.9474 - val_loss: 0.6515 - val_acc: 0.8650\n",
      "Epoch 70/200\n",
      "Learning rate:  0.001\n",
      "196/196 [==============================] - 24s 120ms/step - loss: 0.3489 - acc: 0.9509 - val_loss: 0.6851 - val_acc: 0.8607\n",
      "Epoch 71/200\n",
      "Learning rate:  0.001\n",
      "196/196 [==============================] - 25s 127ms/step - loss: 0.3448 - acc: 0.9518 - val_loss: 0.6225 - val_acc: 0.8799\n",
      "Epoch 72/200\n",
      "Learning rate:  0.001\n",
      "196/196 [==============================] - 24s 124ms/step - loss: 0.3524 - acc: 0.9496 - val_loss: 0.7451 - val_acc: 0.8494\n",
      "Epoch 73/200\n",
      "Learning rate:  0.001\n",
      "196/196 [==============================] - 24s 124ms/step - loss: 0.3454 - acc: 0.9513 - val_loss: 0.7702 - val_acc: 0.8490\n",
      "Epoch 74/200\n",
      "Learning rate:  0.001\n",
      "196/196 [==============================] - 24s 123ms/step - loss: 0.3484 - acc: 0.9502 - val_loss: 0.9710 - val_acc: 0.8144\n",
      "Epoch 75/200\n",
      "Learning rate:  0.001\n",
      "196/196 [==============================] - 24s 123ms/step - loss: 0.3426 - acc: 0.9522 - val_loss: 0.6547 - val_acc: 0.8663\n",
      "Epoch 76/200\n",
      "Learning rate:  0.001\n",
      "196/196 [==============================] - 24s 124ms/step - loss: 0.3457 - acc: 0.9507 - val_loss: 0.7639 - val_acc: 0.8470\n",
      "Epoch 77/200\n",
      "Learning rate:  0.001\n",
      "196/196 [==============================] - 24s 124ms/step - loss: 0.3499 - acc: 0.9491 - val_loss: 0.5868 - val_acc: 0.8822\n",
      "Epoch 78/200\n",
      "Learning rate:  0.001\n",
      "196/196 [==============================] - 24s 124ms/step - loss: 0.3354 - acc: 0.9551 - val_loss: 0.7315 - val_acc: 0.8540\n",
      "Epoch 79/200\n",
      "Learning rate:  0.001\n",
      "196/196 [==============================] - 24s 124ms/step - loss: 0.3396 - acc: 0.9529 - val_loss: 1.0219 - val_acc: 0.8037\n",
      "Epoch 80/200\n",
      "Learning rate:  0.001\n",
      "196/196 [==============================] - 24s 124ms/step - loss: 0.3387 - acc: 0.9525 - val_loss: 0.6519 - val_acc: 0.8766\n",
      "Epoch 81/200\n",
      "Learning rate:  0.001\n",
      "196/196 [==============================] - 24s 124ms/step - loss: 0.3422 - acc: 0.9523 - val_loss: 0.6971 - val_acc: 0.8587\n",
      "Epoch 82/200\n",
      "Learning rate:  0.0001\n",
      "196/196 [==============================] - 24s 124ms/step - loss: 0.2817 - acc: 0.9736 - val_loss: 0.4711 - val_acc: 0.9203\n",
      "Epoch 83/200\n",
      "Learning rate:  0.0001\n",
      "196/196 [==============================] - 24s 124ms/step - loss: 0.2581 - acc: 0.9823 - val_loss: 0.4590 - val_acc: 0.9234\n",
      "Epoch 84/200\n",
      "Learning rate:  0.0001\n",
      "196/196 [==============================] - 24s 124ms/step - loss: 0.2485 - acc: 0.9857 - val_loss: 0.4572 - val_acc: 0.9256\n",
      "Epoch 85/200\n",
      "Learning rate:  0.0001\n",
      "196/196 [==============================] - 24s 124ms/step - loss: 0.2411 - acc: 0.9871 - val_loss: 0.4641 - val_acc: 0.9246\n",
      "Epoch 86/200\n",
      "Learning rate:  0.0001\n",
      "196/196 [==============================] - 24s 124ms/step - loss: 0.2372 - acc: 0.9886 - val_loss: 0.4541 - val_acc: 0.9289\n",
      "Epoch 87/200\n",
      "Learning rate:  0.0001\n",
      "196/196 [==============================] - 24s 124ms/step - loss: 0.2324 - acc: 0.9887 - val_loss: 0.4551 - val_acc: 0.9291\n",
      "Epoch 88/200\n",
      "Learning rate:  0.0001\n",
      "196/196 [==============================] - 24s 123ms/step - loss: 0.2288 - acc: 0.9895 - val_loss: 0.4557 - val_acc: 0.9283\n",
      "Epoch 89/200\n",
      "Learning rate:  0.0001\n",
      "196/196 [==============================] - 24s 124ms/step - loss: 0.2252 - acc: 0.9902 - val_loss: 0.4582 - val_acc: 0.9290\n",
      "Epoch 90/200\n",
      "Learning rate:  0.0001\n",
      "196/196 [==============================] - 24s 123ms/step - loss: 0.2207 - acc: 0.9908 - val_loss: 0.4582 - val_acc: 0.9286\n",
      "Epoch 91/200\n",
      "Learning rate:  0.0001\n",
      "196/196 [==============================] - 24s 124ms/step - loss: 0.2174 - acc: 0.9922 - val_loss: 0.4524 - val_acc: 0.9293\n",
      "Epoch 92/200\n",
      "Learning rate:  0.0001\n",
      "196/196 [==============================] - 24s 123ms/step - loss: 0.2130 - acc: 0.9926 - val_loss: 0.4613 - val_acc: 0.9283\n",
      "Epoch 93/200\n",
      "Learning rate:  0.0001\n",
      "196/196 [==============================] - 24s 124ms/step - loss: 0.2107 - acc: 0.9929 - val_loss: 0.4636 - val_acc: 0.9298\n",
      "Epoch 94/200\n",
      "Learning rate:  0.0001\n",
      "196/196 [==============================] - 24s 123ms/step - loss: 0.2082 - acc: 0.9931 - val_loss: 0.4698 - val_acc: 0.9282\n",
      "Epoch 95/200\n",
      "Learning rate:  0.0001\n",
      "196/196 [==============================] - 24s 122ms/step - loss: 0.2069 - acc: 0.9926 - val_loss: 0.4841 - val_acc: 0.9224\n",
      "Epoch 96/200\n",
      "Learning rate:  0.0001\n",
      "196/196 [==============================] - 24s 124ms/step - loss: 0.2019 - acc: 0.9934 - val_loss: 0.4746 - val_acc: 0.9274\n",
      "Epoch 97/200\n",
      "Learning rate:  0.0001\n",
      "196/196 [==============================] - 24s 124ms/step - loss: 0.2009 - acc: 0.9931 - val_loss: 0.4636 - val_acc: 0.9293\n",
      "Epoch 98/200\n",
      "Learning rate:  0.0001\n",
      "196/196 [==============================] - 24s 124ms/step - loss: 0.1971 - acc: 0.9938 - val_loss: 0.4705 - val_acc: 0.9296\n",
      "Epoch 99/200\n",
      "Learning rate:  0.0001\n",
      "196/196 [==============================] - 24s 125ms/step - loss: 0.1961 - acc: 0.9936 - val_loss: 0.4771 - val_acc: 0.9261\n",
      "Epoch 100/200\n",
      "Learning rate:  0.0001\n",
      "196/196 [==============================] - 24s 121ms/step - loss: 0.1941 - acc: 0.9943 - val_loss: 0.4742 - val_acc: 0.9262\n",
      "Epoch 101/200\n",
      "Learning rate:  0.0001\n",
      "196/196 [==============================] - 24s 121ms/step - loss: 0.1909 - acc: 0.9944 - val_loss: 0.4766 - val_acc: 0.9276\n",
      "Epoch 102/200\n",
      "Learning rate:  0.0001\n",
      "196/196 [==============================] - 24s 121ms/step - loss: 0.1881 - acc: 0.9951 - val_loss: 0.4723 - val_acc: 0.9294\n",
      "Epoch 103/200\n",
      "Learning rate:  0.0001\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "196/196 [==============================] - 23s 118ms/step - loss: 0.1861 - acc: 0.9952 - val_loss: 0.4957 - val_acc: 0.9231\n",
      "Epoch 104/200\n",
      "Learning rate:  0.0001\n",
      "196/196 [==============================] - 23s 118ms/step - loss: 0.1855 - acc: 0.9946 - val_loss: 0.4881 - val_acc: 0.9266\n",
      "Epoch 105/200\n",
      "Learning rate:  0.0001\n",
      "196/196 [==============================] - 23s 119ms/step - loss: 0.1837 - acc: 0.9950 - val_loss: 0.5001 - val_acc: 0.9243\n",
      "Epoch 106/200\n",
      "Learning rate:  0.0001\n",
      "196/196 [==============================] - 23s 118ms/step - loss: 0.1800 - acc: 0.9956 - val_loss: 0.4779 - val_acc: 0.9279\n",
      "Epoch 107/200\n",
      "Learning rate:  0.0001\n",
      "196/196 [==============================] - 23s 118ms/step - loss: 0.1791 - acc: 0.9950 - val_loss: 0.4881 - val_acc: 0.9234\n",
      "Epoch 108/200\n",
      "Learning rate:  0.0001\n",
      "196/196 [==============================] - 23s 119ms/step - loss: 0.1782 - acc: 0.9950 - val_loss: 0.4930 - val_acc: 0.9252\n",
      "Epoch 109/200\n",
      "Learning rate:  0.0001\n",
      "196/196 [==============================] - 23s 118ms/step - loss: 0.1758 - acc: 0.9956 - val_loss: 0.4772 - val_acc: 0.9254\n",
      "Epoch 110/200\n",
      "Learning rate:  0.0001\n",
      "196/196 [==============================] - 23s 118ms/step - loss: 0.1737 - acc: 0.9957 - val_loss: 0.4755 - val_acc: 0.9285\n",
      "Epoch 111/200\n",
      "Learning rate:  0.0001\n",
      "196/196 [==============================] - 23s 119ms/step - loss: 0.1734 - acc: 0.9954 - val_loss: 0.4914 - val_acc: 0.9231\n",
      "Epoch 112/200\n",
      "Learning rate:  0.0001\n",
      "196/196 [==============================] - 23s 118ms/step - loss: 0.1706 - acc: 0.9961 - val_loss: 0.4819 - val_acc: 0.9273\n",
      "Epoch 113/200\n",
      "Learning rate:  0.0001\n",
      "196/196 [==============================] - 23s 118ms/step - loss: 0.1706 - acc: 0.9950 - val_loss: 0.5106 - val_acc: 0.9227\n",
      "Epoch 114/200\n",
      "Learning rate:  0.0001\n",
      "196/196 [==============================] - 23s 118ms/step - loss: 0.1668 - acc: 0.9961 - val_loss: 0.4821 - val_acc: 0.9283\n",
      "Epoch 115/200\n",
      "Learning rate:  0.0001\n",
      "196/196 [==============================] - 23s 119ms/step - loss: 0.1651 - acc: 0.9962 - val_loss: 0.4737 - val_acc: 0.9292\n",
      "Epoch 116/200\n",
      "Learning rate:  0.0001\n",
      "196/196 [==============================] - 23s 119ms/step - loss: 0.1645 - acc: 0.9959 - val_loss: 0.4778 - val_acc: 0.9291\n",
      "Epoch 117/200\n",
      "Learning rate:  0.0001\n",
      "196/196 [==============================] - 23s 119ms/step - loss: 0.1639 - acc: 0.9960 - val_loss: 0.4849 - val_acc: 0.9285\n",
      "Epoch 118/200\n",
      "Learning rate:  0.0001\n",
      "196/196 [==============================] - 23s 118ms/step - loss: 0.1614 - acc: 0.9963 - val_loss: 0.4753 - val_acc: 0.9283\n",
      "Epoch 119/200\n",
      "Learning rate:  0.0001\n",
      "196/196 [==============================] - 23s 119ms/step - loss: 0.1612 - acc: 0.9957 - val_loss: 0.4971 - val_acc: 0.9257\n",
      "Epoch 120/200\n",
      "Learning rate:  0.0001\n",
      "196/196 [==============================] - 23s 119ms/step - loss: 0.1602 - acc: 0.9961 - val_loss: 0.4855 - val_acc: 0.9289\n",
      "Epoch 121/200\n",
      "Learning rate:  0.0001\n",
      "196/196 [==============================] - 23s 119ms/step - loss: 0.1594 - acc: 0.9959 - val_loss: 0.4775 - val_acc: 0.9271\n",
      "Epoch 122/200\n",
      "Learning rate:  1e-05\n",
      "196/196 [==============================] - 23s 119ms/step - loss: 0.1554 - acc: 0.9972 - val_loss: 0.4702 - val_acc: 0.9281\n",
      "Epoch 123/200\n",
      "Learning rate:  1e-05\n",
      "196/196 [==============================] - 23s 119ms/step - loss: 0.1545 - acc: 0.9973 - val_loss: 0.4691 - val_acc: 0.9279\n",
      "Epoch 124/200\n",
      "Learning rate:  1e-05\n",
      "196/196 [==============================] - 23s 119ms/step - loss: 0.1530 - acc: 0.9980 - val_loss: 0.4688 - val_acc: 0.9269\n",
      "Epoch 125/200\n",
      "Learning rate:  1e-05\n",
      "196/196 [==============================] - 23s 118ms/step - loss: 0.1529 - acc: 0.9978 - val_loss: 0.4650 - val_acc: 0.9293\n",
      "Epoch 126/200\n",
      "Learning rate:  1e-05\n",
      "196/196 [==============================] - 23s 119ms/step - loss: 0.1533 - acc: 0.9977 - val_loss: 0.4664 - val_acc: 0.9297\n",
      "Epoch 127/200\n",
      "Learning rate:  1e-05\n",
      "196/196 [==============================] - 23s 119ms/step - loss: 0.1523 - acc: 0.9979 - val_loss: 0.4660 - val_acc: 0.9287\n",
      "Epoch 128/200\n",
      "Learning rate:  1e-05\n",
      "196/196 [==============================] - 23s 119ms/step - loss: 0.1522 - acc: 0.9980 - val_loss: 0.4657 - val_acc: 0.9290\n",
      "Epoch 129/200\n",
      "Learning rate:  1e-05\n",
      "196/196 [==============================] - 23s 120ms/step - loss: 0.1518 - acc: 0.9982 - val_loss: 0.4651 - val_acc: 0.9299\n",
      "Epoch 130/200\n",
      "Learning rate:  1e-05\n",
      "196/196 [==============================] - 24s 121ms/step - loss: 0.1517 - acc: 0.9983 - val_loss: 0.4654 - val_acc: 0.9298\n",
      "Epoch 131/200\n",
      "Learning rate:  1e-05\n",
      "196/196 [==============================] - 24s 121ms/step - loss: 0.1511 - acc: 0.9983 - val_loss: 0.4668 - val_acc: 0.9298\n",
      "Epoch 132/200\n",
      "Learning rate:  1e-05\n",
      "196/196 [==============================] - 24s 121ms/step - loss: 0.1508 - acc: 0.9982 - val_loss: 0.4657 - val_acc: 0.9297\n",
      "Epoch 133/200\n",
      "Learning rate:  1e-05\n",
      "196/196 [==============================] - 24s 121ms/step - loss: 0.1502 - acc: 0.9984 - val_loss: 0.4671 - val_acc: 0.9300\n",
      "Epoch 134/200\n",
      "Learning rate:  1e-05\n",
      "196/196 [==============================] - 24s 120ms/step - loss: 0.1505 - acc: 0.9981 - val_loss: 0.4670 - val_acc: 0.9294\n",
      "Epoch 135/200\n",
      "Learning rate:  1e-05\n",
      "196/196 [==============================] - 23s 120ms/step - loss: 0.1498 - acc: 0.9986 - val_loss: 0.4675 - val_acc: 0.9292\n",
      "Epoch 136/200\n",
      "Learning rate:  1e-05\n",
      "196/196 [==============================] - 24s 121ms/step - loss: 0.1507 - acc: 0.9980 - val_loss: 0.4648 - val_acc: 0.9306\n",
      "Epoch 137/200\n",
      "Learning rate:  1e-05\n",
      "196/196 [==============================] - 24s 121ms/step - loss: 0.1492 - acc: 0.9985 - val_loss: 0.4652 - val_acc: 0.9296\n",
      "Epoch 138/200\n",
      "Learning rate:  1e-05\n",
      "196/196 [==============================] - 24s 120ms/step - loss: 0.1493 - acc: 0.9985 - val_loss: 0.4657 - val_acc: 0.9301\n",
      "Epoch 139/200\n",
      "Learning rate:  1e-05\n",
      "196/196 [==============================] - 24s 121ms/step - loss: 0.1488 - acc: 0.9982 - val_loss: 0.4667 - val_acc: 0.9298\n",
      "Epoch 140/200\n",
      "Learning rate:  1e-05\n",
      "196/196 [==============================] - 24s 121ms/step - loss: 0.1486 - acc: 0.9985 - val_loss: 0.4655 - val_acc: 0.9300\n",
      "Epoch 141/200\n",
      "Learning rate:  1e-05\n",
      "196/196 [==============================] - 24s 121ms/step - loss: 0.1484 - acc: 0.9986 - val_loss: 0.4663 - val_acc: 0.9307\n",
      "Epoch 142/200\n",
      "Learning rate:  1e-05\n",
      "196/196 [==============================] - 24s 120ms/step - loss: 0.1483 - acc: 0.9984 - val_loss: 0.4679 - val_acc: 0.9305\n",
      "Epoch 143/200\n",
      "Learning rate:  1e-05\n",
      "196/196 [==============================] - 24s 121ms/step - loss: 0.1487 - acc: 0.9983 - val_loss: 0.4658 - val_acc: 0.9310\n",
      "Epoch 144/200\n",
      "Learning rate:  1e-05\n",
      "196/196 [==============================] - 24s 121ms/step - loss: 0.1478 - acc: 0.9984 - val_loss: 0.4638 - val_acc: 0.9318\n",
      "Epoch 145/200\n",
      "Learning rate:  1e-05\n",
      "196/196 [==============================] - 24s 124ms/step - loss: 0.1481 - acc: 0.9982 - val_loss: 0.4669 - val_acc: 0.9310\n",
      "Epoch 146/200\n",
      "Learning rate:  1e-05\n",
      "196/196 [==============================] - 24s 120ms/step - loss: 0.1471 - acc: 0.9987 - val_loss: 0.4679 - val_acc: 0.9297\n",
      "Epoch 147/200\n",
      "Learning rate:  1e-05\n",
      "196/196 [==============================] - 24s 121ms/step - loss: 0.1471 - acc: 0.9986 - val_loss: 0.4673 - val_acc: 0.9307\n",
      "Epoch 148/200\n",
      "Learning rate:  1e-05\n",
      "196/196 [==============================] - 24s 121ms/step - loss: 0.1471 - acc: 0.9986 - val_loss: 0.4701 - val_acc: 0.9303\n",
      "Epoch 149/200\n",
      "Learning rate:  1e-05\n",
      "196/196 [==============================] - 24s 120ms/step - loss: 0.1467 - acc: 0.9985 - val_loss: 0.4700 - val_acc: 0.9304\n",
      "Epoch 150/200\n",
      "Learning rate:  1e-05\n",
      "196/196 [==============================] - 24s 121ms/step - loss: 0.1465 - acc: 0.9985 - val_loss: 0.4693 - val_acc: 0.9300\n",
      "Epoch 151/200\n",
      "Learning rate:  1e-05\n",
      "196/196 [==============================] - 24s 121ms/step - loss: 0.1465 - acc: 0.9985 - val_loss: 0.4675 - val_acc: 0.9306\n",
      "Epoch 152/200\n",
      "Learning rate:  1e-05\n",
      "196/196 [==============================] - 24s 121ms/step - loss: 0.1460 - acc: 0.9987 - val_loss: 0.4675 - val_acc: 0.9306\n",
      "Epoch 153/200\n",
      "Learning rate:  1e-05\n",
      "196/196 [==============================] - 23s 120ms/step - loss: 0.1459 - acc: 0.9986 - val_loss: 0.4699 - val_acc: 0.9305\n",
      "Epoch 154/200\n",
      "Learning rate:  1e-05\n",
      "196/196 [==============================] - 24s 122ms/step - loss: 0.1451 - acc: 0.9989 - val_loss: 0.4699 - val_acc: 0.9312\n",
      "Epoch 155/200\n",
      "Learning rate:  1e-05\n",
      "196/196 [==============================] - 24s 120ms/step - loss: 0.1451 - acc: 0.9987 - val_loss: 0.4681 - val_acc: 0.9310\n",
      "Epoch 156/200\n",
      "Learning rate:  1e-05\n",
      "196/196 [==============================] - 24s 121ms/step - loss: 0.1452 - acc: 0.9986 - val_loss: 0.4697 - val_acc: 0.9301\n",
      "Epoch 157/200\n",
      "Learning rate:  1e-05\n",
      "196/196 [==============================] - 24s 121ms/step - loss: 0.1451 - acc: 0.9985 - val_loss: 0.4684 - val_acc: 0.9307\n",
      "Epoch 158/200\n",
      "Learning rate:  1e-05\n",
      "196/196 [==============================] - 24s 120ms/step - loss: 0.1446 - acc: 0.9988 - val_loss: 0.4700 - val_acc: 0.9299\n",
      "Epoch 159/200\n",
      "Learning rate:  1e-05\n",
      "196/196 [==============================] - 24s 121ms/step - loss: 0.1453 - acc: 0.9984 - val_loss: 0.4692 - val_acc: 0.9321\n",
      "Epoch 160/200\n",
      "Learning rate:  1e-05\n",
      "196/196 [==============================] - 24s 121ms/step - loss: 0.1445 - acc: 0.9986 - val_loss: 0.4676 - val_acc: 0.9312\n",
      "Epoch 161/200\n",
      "Learning rate:  1e-05\n",
      "196/196 [==============================] - 24s 121ms/step - loss: 0.1436 - acc: 0.9990 - val_loss: 0.4706 - val_acc: 0.9327\n",
      "Epoch 162/200\n",
      "Learning rate:  1e-06\n",
      "196/196 [==============================] - 24s 121ms/step - loss: 0.1440 - acc: 0.9984 - val_loss: 0.4713 - val_acc: 0.9329\n",
      "Epoch 163/200\n",
      "Learning rate:  1e-06\n",
      "196/196 [==============================] - 24s 120ms/step - loss: 0.1440 - acc: 0.9989 - val_loss: 0.4703 - val_acc: 0.9328\n",
      "Epoch 164/200\n",
      "Learning rate:  1e-06\n",
      "196/196 [==============================] - 24s 121ms/step - loss: 0.1437 - acc: 0.9989 - val_loss: 0.4705 - val_acc: 0.9324\n",
      "Epoch 165/200\n",
      "Learning rate:  1e-06\n",
      "196/196 [==============================] - 24s 121ms/step - loss: 0.1438 - acc: 0.9988 - val_loss: 0.4712 - val_acc: 0.9321\n",
      "Epoch 166/200\n",
      "Learning rate:  1e-06\n",
      "196/196 [==============================] - 24s 121ms/step - loss: 0.1440 - acc: 0.9986 - val_loss: 0.4699 - val_acc: 0.9322\n",
      "Epoch 167/200\n",
      "Learning rate:  1e-06\n",
      "196/196 [==============================] - 24s 121ms/step - loss: 0.1437 - acc: 0.9988 - val_loss: 0.4700 - val_acc: 0.9323\n",
      "Epoch 168/200\n",
      "Learning rate:  1e-06\n",
      "196/196 [==============================] - 24s 121ms/step - loss: 0.1438 - acc: 0.9989 - val_loss: 0.4699 - val_acc: 0.9327\n",
      "Epoch 169/200\n",
      "Learning rate:  1e-06\n",
      "196/196 [==============================] - 24s 120ms/step - loss: 0.1436 - acc: 0.9988 - val_loss: 0.4697 - val_acc: 0.9319\n",
      "Epoch 170/200\n",
      "Learning rate:  1e-06\n",
      "196/196 [==============================] - 24s 121ms/step - loss: 0.1433 - acc: 0.9989 - val_loss: 0.4694 - val_acc: 0.9319\n",
      "Epoch 171/200\n",
      "Learning rate:  1e-06\n",
      "196/196 [==============================] - 24s 122ms/step - loss: 0.1436 - acc: 0.9988 - val_loss: 0.4696 - val_acc: 0.9321\n",
      "Epoch 172/200\n",
      "Learning rate:  1e-06\n",
      "196/196 [==============================] - 24s 121ms/step - loss: 0.1432 - acc: 0.9990 - val_loss: 0.4694 - val_acc: 0.9320\n",
      "Epoch 173/200\n",
      "Learning rate:  1e-06\n",
      "196/196 [==============================] - 24s 121ms/step - loss: 0.1435 - acc: 0.9989 - val_loss: 0.4686 - val_acc: 0.9324\n",
      "Epoch 174/200\n",
      "Learning rate:  1e-06\n",
      "196/196 [==============================] - 24s 120ms/step - loss: 0.1432 - acc: 0.9988 - val_loss: 0.4694 - val_acc: 0.9326\n",
      "Epoch 175/200\n",
      "Learning rate:  1e-06\n",
      "196/196 [==============================] - 24s 121ms/step - loss: 0.1434 - acc: 0.9988 - val_loss: 0.4686 - val_acc: 0.9327\n",
      "Epoch 176/200\n",
      "Learning rate:  1e-06\n",
      "196/196 [==============================] - 24s 121ms/step - loss: 0.1440 - acc: 0.9987 - val_loss: 0.4686 - val_acc: 0.9323\n",
      "Epoch 177/200\n",
      "Learning rate:  1e-06\n",
      "196/196 [==============================] - 24s 121ms/step - loss: 0.1434 - acc: 0.9989 - val_loss: 0.4686 - val_acc: 0.9326\n",
      "Epoch 178/200\n",
      "Learning rate:  1e-06\n",
      "196/196 [==============================] - 24s 121ms/step - loss: 0.1433 - acc: 0.9989 - val_loss: 0.4689 - val_acc: 0.9314\n",
      "Epoch 179/200\n",
      "Learning rate:  1e-06\n",
      "196/196 [==============================] - 24s 121ms/step - loss: 0.1433 - acc: 0.9988 - val_loss: 0.4692 - val_acc: 0.9317\n",
      "Epoch 180/200\n",
      "Learning rate:  1e-06\n",
      "196/196 [==============================] - 24s 121ms/step - loss: 0.1439 - acc: 0.9986 - val_loss: 0.4684 - val_acc: 0.9314\n",
      "Epoch 181/200\n",
      "Learning rate:  1e-06\n",
      "196/196 [==============================] - 24s 121ms/step - loss: 0.1433 - acc: 0.9988 - val_loss: 0.4692 - val_acc: 0.9320\n",
      "Epoch 182/200\n",
      "Learning rate:  5e-07\n",
      "196/196 [==============================] - 24s 121ms/step - loss: 0.1431 - acc: 0.9989 - val_loss: 0.4687 - val_acc: 0.9324\n",
      "Epoch 183/200\n",
      "Learning rate:  5e-07\n",
      "196/196 [==============================] - 24s 121ms/step - loss: 0.1432 - acc: 0.9988 - val_loss: 0.4695 - val_acc: 0.9319\n",
      "Epoch 184/200\n",
      "Learning rate:  5e-07\n",
      "196/196 [==============================] - 24s 121ms/step - loss: 0.1433 - acc: 0.9988 - val_loss: 0.4690 - val_acc: 0.9322\n",
      "Epoch 185/200\n",
      "Learning rate:  5e-07\n",
      "196/196 [==============================] - 24s 121ms/step - loss: 0.1430 - acc: 0.9990 - val_loss: 0.4690 - val_acc: 0.9319\n",
      "Epoch 186/200\n",
      "Learning rate:  5e-07\n",
      "196/196 [==============================] - 24s 120ms/step - loss: 0.1437 - acc: 0.9987 - val_loss: 0.4693 - val_acc: 0.9318\n",
      "Epoch 187/200\n",
      "Learning rate:  5e-07\n",
      "196/196 [==============================] - 24s 120ms/step - loss: 0.1430 - acc: 0.9989 - val_loss: 0.4693 - val_acc: 0.9319\n",
      "Epoch 188/200\n",
      "Learning rate:  5e-07\n",
      "196/196 [==============================] - 24s 121ms/step - loss: 0.1435 - acc: 0.9987 - val_loss: 0.4691 - val_acc: 0.9321\n",
      "Epoch 189/200\n",
      "Learning rate:  5e-07\n",
      "196/196 [==============================] - 24s 121ms/step - loss: 0.1431 - acc: 0.9988 - val_loss: 0.4689 - val_acc: 0.9318\n",
      "Epoch 190/200\n",
      "Learning rate:  5e-07\n",
      "196/196 [==============================] - 24s 120ms/step - loss: 0.1435 - acc: 0.9988 - val_loss: 0.4684 - val_acc: 0.9319\n",
      "Epoch 191/200\n",
      "Learning rate:  5e-07\n",
      "196/196 [==============================] - 24s 121ms/step - loss: 0.1432 - acc: 0.9989 - val_loss: 0.4690 - val_acc: 0.9318\n",
      "Epoch 192/200\n",
      "Learning rate:  5e-07\n",
      "196/196 [==============================] - 24s 121ms/step - loss: 0.1428 - acc: 0.9992 - val_loss: 0.4697 - val_acc: 0.9315\n",
      "Epoch 193/200\n",
      "Learning rate:  5e-07\n",
      "196/196 [==============================] - 24s 120ms/step - loss: 0.1433 - acc: 0.9990 - val_loss: 0.4695 - val_acc: 0.9318\n",
      "Epoch 194/200\n",
      "Learning rate:  5e-07\n",
      "196/196 [==============================] - 24s 120ms/step - loss: 0.1431 - acc: 0.9990 - val_loss: 0.4693 - val_acc: 0.9321\n",
      "Epoch 195/200\n",
      "Learning rate:  5e-07\n",
      "196/196 [==============================] - 24s 121ms/step - loss: 0.1434 - acc: 0.9986 - val_loss: 0.4692 - val_acc: 0.9321\n",
      "Epoch 196/200\n",
      "Learning rate:  5e-07\n",
      "196/196 [==============================] - 24s 121ms/step - loss: 0.1435 - acc: 0.9987 - val_loss: 0.4694 - val_acc: 0.9321\n",
      "Epoch 197/200\n",
      "Learning rate:  5e-07\n",
      "196/196 [==============================] - 24s 120ms/step - loss: 0.1431 - acc: 0.9989 - val_loss: 0.4687 - val_acc: 0.9321\n",
      "Epoch 198/200\n",
      "Learning rate:  5e-07\n",
      "196/196 [==============================] - 24s 121ms/step - loss: 0.1431 - acc: 0.9990 - val_loss: 0.4691 - val_acc: 0.9322\n",
      "Epoch 199/200\n",
      "Learning rate:  5e-07\n",
      "196/196 [==============================] - 24s 121ms/step - loss: 0.1436 - acc: 0.9986 - val_loss: 0.4693 - val_acc: 0.9323\n",
      "Epoch 200/200\n",
      "Learning rate:  5e-07\n",
      "196/196 [==============================] - 24s 121ms/step - loss: 0.1428 - acc: 0.9990 - val_loss: 0.4687 - val_acc: 0.9321\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f02cc6092e8>"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch_size = 256\n",
    "model.fit_generator(datagen.flow(x_train, y_train, batch_size=batch_size),\n",
    "                        validation_data=(x_test, y_test),\n",
    "                        epochs=epochs,\n",
    "                        verbose=1, \n",
    "                        workers=4,\n",
    "                        callbacks=callbacks)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4.模型评估"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Score trained model.\n",
    "scores = model.evaluate(x_test, y_test, verbose=1)\n",
    "print('Test loss:', scores[0])\n",
    "print('Test accuracy:', scores[1])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
